{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Predicting Movie Review Sentriment with BERT\n",
    "This example is slight different with the tutorial in [**google's BERT**](https://github.com/google-research/bert#pre-trained-models) which uses pre-trained bert model. Some codes are borrowed from [this tutorial](https://github.com/google-research/bert/blob/master/predicting_movie_reviews_with_bert_on_tf_hub.ipynb).\n",
    "\n",
    "The purpose of this tutorial is telling you how to use **BERT_with_keras** to pretrain an encoding model and train a classification model. Let's get started!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import tensorflow as tf\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data\n",
    "\n",
    "Let's use Standord's Large Movie Review Dataset for BERT pretraining and fine-tuning, the code below, which downloads,extracts and imports the dateset, is borrowed from this [tensorflow tutorial](https://www.tensorflow.org/hub/tutorials/text_classification_with_tf_hub). The dataset consists of IMDB movie reviews labeled by positivity from 1 to 10."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Load all files from a directory in a DataFrame.\n",
    "def load_directory_data(directory):\n",
    "    data = {}\n",
    "    data[\"sentence\"] = []\n",
    "    data[\"sentiment\"] = []\n",
    "    for file_path in os.listdir(directory):\n",
    "        with tf.gfile.GFile(os.path.join(directory, file_path), \"r\") as f:\n",
    "            data[\"sentence\"].append(f.read())\n",
    "            data[\"sentiment\"].append(re.match(\"\\d+_(\\d+)\\.txt\", file_path).group(1))\n",
    "    return pd.DataFrame.from_dict(data)\n",
    "\n",
    "# Merge positive and negative examples, add a polarity column and shuffle.\n",
    "def load_dataset(directory):\n",
    "    pos_df = load_directory_data(os.path.join(directory, \"pos\"))\n",
    "    neg_df = load_directory_data(os.path.join(directory, \"neg\"))\n",
    "    pos_df[\"polarity\"] = 1\n",
    "    neg_df[\"polarity\"] = 0\n",
    "    return pd.concat([pos_df, neg_df]).sample(frac=1).reset_index(drop=True)\n",
    "\n",
    "# Download and process the dataset files.\n",
    "def download_and_load_datasets(force_download=False):\n",
    "    dataset = tf.keras.utils.get_file(\n",
    "        fname=\"aclImdb.tar.gz\", \n",
    "        origin=\"http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz\", \n",
    "        extract=True)\n",
    "  \n",
    "    train_df = load_dataset(os.path.join(os.path.dirname(dataset), \n",
    "                                         \"aclImdb\", \"train\"))\n",
    "    test_df = load_dataset(os.path.join(os.path.dirname(dataset), \n",
    "                                          \"aclImdb\", \"test\"))\n",
    "    return train_df, test_df\n",
    " \n",
    "train, test = download_and_load_datasets()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>polarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I borrowed this movie from library think it mi...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\"Private Practice\" is being spun off the fairl...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Wesley Snipes is James Dial, an assassin for h...</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>There is certainly emotion between the two mai...</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>there are three kinds of bad films - the cheap...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            sentence sentiment  polarity\n",
       "0  I borrowed this movie from library think it mi...         1         0\n",
       "1  \"Private Practice\" is being spun off the fairl...         3         0\n",
       "2  Wesley Snipes is James Dial, an assassin for h...         4         0\n",
       "3  There is certainly emotion between the two mai...         7         1\n",
       "4  there are three kinds of bad films - the cheap...         1         0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pre-training\n",
    "Let's use the IMDB movie reviews data for bert's pretraining. When you want to get a better **bert pre-training model**, you need to prepare data by yourself."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import spacy\n",
    "import random\n",
    "from const import bert_data_path,bert_model_path\n",
    "from preprocess import create_pretraining_data_from_docs\n",
    "from pretraining import bert_pretraining"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Before pre-training bert model, we need to transform the movie revies data to the format we use in **Bert_with_keras model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num-0: tokens: [CLS] we also have lots of scenes with the hero fl ##au ##nting all the [MASK] of respects and protocol which the rest of the tibetan [MASK] accord ##s the dalai lama , [MASK] as we [MASK] that the hero has deep and profound [MASK] ##erence [MASK] these people and their spiritual [MASK] . [SEP] that this guy is now [MASK] buddhist , sort of , in his own way , even though we ourselves don ##´ ##t seem to know what his transformation en ##tails or how [MASK] [MASK] [MASK] it to go . and last but not least , we hang a stat ##istic onto the [MASK] [MASK] the film about [MASK] app ##all ##ingly the chinese have treated the tibetan ##s ( which is [SEP]\n",
      "segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "is_random_next: False\n",
      "masked_lm_positions: 9 15 26 33 36 44 45 46 52 60 70 88 89 90 97 107 108 109 113\n",
      "masked_lm_labels: hero marks society even pretend rev ##erence for leader a , far we want but the end of how\n",
      "\n",
      "\n",
      "num-1: tokens: [CLS] i [MASK] people writing about how great this movie was . it was horrible ! the acting [MASK] [MASK] - [MASK] at best [MASK] it made a lot of money because teenage girls went to see the movie 7 times in the theaters because of leonardo . where [MASK] hell did they get the money ? [SEP] it sank , [MASK] was running through a lot of people ' s minds effective maybe even a little [MASK] [MASK] . does anyone realize that certain restored didn ' t [MASK] [MASK] the ship because there was a fire on [MASK] before it even took off ? no , you don ' [MASK] because all you [MASK] [MASK] a rich girl falling for a poor boy and he [SEP]\n",
      "segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "is_random_next: False\n",
      "masked_lm_positions: 2 18 19 21 24 25 49 61 72 77 78 85 89 90 92 99 111 115 116\n",
      "masked_lm_labels: see was sub par . it the what ; conspiracy stuff people even board ship board t see is\n",
      "\n",
      "\n",
      "num-2: tokens: [CLS] any ##how , [MASK] ' s an entertaining film [MASK] if you ' ve [MASK] nothing to do on a weekday evening . [SEP] york segment of ' the day after tomorrow ' , but that stalk ' t [MASK] it any less of a film . [MASK] , the script [MASK] [MASK] ' t there . it ' ##ang merely functional , flat , and lacking [MASK] [MASK] . great british talents like [MASK] carly ##sle and david such ##et to name but two do their level best with [MASK] they ' ve got [MASK] but [MASK] characters are two - dimensional cy ##pher ##s , ##skin something out of [MASK] old marvel comic . [MASK] it ' d be frankly easier to turn back [SEP]\n",
      "segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "is_random_next: True\n",
      "masked_lm_positions: 4 10 15 37 40 48 52 53 60 68 69 75 84 91 96 98 108 112 117\n",
      "masked_lm_labels: it , got shouldn make however just isn s in depth robert but what , their like an and\n",
      "\n",
      "\n",
      "num-3: tokens: [CLS] directed by jacques tour ##ne ##ur ( cat people , out of the past , night of the gemini ) and [MASK] by phillip dunne ( how green was my valley ) [MASK] of the indies is a quite interesting adventure pirate movie . [SEP] pirate anne [MASK] [MASK] who actually lived and sailed through 18th century ' s atlantic . < br [MASK] > < br / > the film begins with the sea battle where anne ' s ( jean peters og [MASK] ship attacks [MASK] trade partisan that was [MASK] its way to europe from the south [MASK] . as a [MASK] a treasure [MASK] great value is captured along with a handsome french officer pierre la [MASK] ( louis jo [MASK] ##dan [SEP]\n",
      "segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "is_random_next: False\n",
      "masked_lm_positions: 19 22 30 31 33 48 49 64 84 85 88 90 93 101 105 108 118 121 125\n",
      "masked_lm_labels: demon written my valley anne bone ##y / ) pirate a ship on america result of officer rochelle ##ur\n",
      "\n",
      "\n",
      "num-4: tokens: [CLS] / > obviously it ' 1745 not an [MASK] contender , [MASK] it is entertaining and serves [MASK] purpose . < [MASK] / > < br [SEP] [MASK] > and for the men out there [MASK] there were an equal share [MASK] [MASK] and women in the movie theater ( most were with their wives / girlfriends ) , and the men were [MASK] fact laughing . : ) [SEP]\n",
      "segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "is_random_next: False\n",
      "masked_lm_positions: 6 9 12 18 22 28 36 42 43 56 64\n",
      "masked_lm_labels: s oscar but its br / , of men / in\n",
      "\n",
      "\n",
      "num-5: tokens: [CLS] bbc ' s [MASK] hour adaptation of the novel by [MASK] [MASK] . . . \" fingers ##mith \" ##anies life is tough without money , especially in dickens ##ian london . dark [MASK] lead to [MASK] ##pic ##able [MASK] ##s . is love really [MASK] [MASK] luxury for [MASK] [MASK] and free ? ? [SEP] as \" sue tri ##nder [MASK] radius give [MASK] performances as the leading ladies asking this question . . . of each other . . . whilst [MASK] evans shine ##s as the delightful ##ly [MASK] [MASK] gentleman \" . . with great support from im ##eld ##a st ##au ##nton ' s \" mrs sucks ##by \" , david trough ##ton ' s \" mr ib ##bs \" and [SEP]\n",
      "segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "is_random_next: False\n",
      "masked_lm_positions: 4 11 12 20 26 34 37 40 46 47 50 51 62 63 65 84 92 93 106\n",
      "masked_lm_labels: 3 sarah waters . , deeds des dilemma just a the rich \" both fantastic rupert bad \" ##au\n",
      "\n",
      "\n",
      "num-6: tokens: [CLS] the women is a cute movie about women at [MASK] ages ( but mostly 30 + ) and their issues in life . [SEP] in [MASK] ##lity bernstein but also about relationships with friends [MASK] family [MASK] making time to connect with others , [MASK] [MASK] image [MASK] com ##promising your values , [MASK] in life , and finding what will really make you happy . < br / > < br / > it ' s also about being [MASK] to yourself . a lot of [MASK] , people will give you advice but not really [MASK] it [MASK] . sometimes they have created a [MASK] ##usion for themselves , and should succeeding [MASK] follow in that same path or react [MASK] on your feelings [SEP]\n",
      "segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "is_random_next: False\n",
      "masked_lm_positions: 10 26 28 35 37 45 46 48 54 61 71 81 88 98 100 107 114 115 123\n",
      "masked_lm_labels: all ##fide , & , problems with , accomplishments will < true times follow themselves del you really based\n",
      "\n",
      "\n",
      "num-7: tokens: [CLS] this movie works because it feels so genuine . [MASK] story is simple and realistic , perfectly capturing [MASK] joy ##s and an ##xie ##ties of adolescent [MASK] [MASK] [MASK] that most / all of [MASK] [MASK] during our teen years . < br / > < [MASK] [SEP] and stefan the heroic vampire is about as charming as a ref [MASK] [MASK] ##ated fireplace poker , but who cares ? [MASK] ' s [MASK] one reason to watch this movie , and his name levy [MASK] ##du ! he ' [MASK] a physical homage [MASK] nos ##fera ##tu and he has the best lines in the movie [MASK] all spoken in the ras ##py voice of a man who smoke ##s ten packs of cigarettes [SEP]\n",
      "segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "is_random_next: True\n",
      "masked_lm_positions: 3 10 19 28 29 30 36 37 48 53 62 63 72 75 86 87 92 96 109\n",
      "masked_lm_labels: works the the love and sexuality us experienced br heroic ##ri ##ger there only is ra s to ,\n",
      "\n",
      "\n",
      "num-8: tokens: [CLS] they repeat pointless \" special effects \" so many times that it ' s obvious [MASK] were just trying to cover up the fact that they only shot 30 minutes of footage . if i were forced to [MASK] [MASK] movie on repeat [MASK] would [MASK] [MASK] ##on [MASK] unconscious with my [MASK] hands after [MASK] one and a half times through . [SEP] abu as his dog . the prince has fallen into a sleep and nothing unexpected wake her [MASK] so ja ##ffa ##r sends his servant hal ##ima for ahmad and the dog , in hopes the prince can rouse [MASK] [MASK] he does awake ##n [MASK] . she boards [MASK] ship to find [MASK] doctor to [MASK] ahmad , but [MASK] is [SEP]\n",
      "segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "is_random_next: True\n",
      "masked_lm_positions: 16 39 40 44 46 47 49 53 56 66 79 82 104 105 110 114 118 121 125\n",
      "masked_lm_labels: they watch this i blu ##dge myself own about as can . her . her a a cure she\n",
      "\n",
      "\n",
      "num-9: tokens: [CLS] spy when men clear the way so none will [MASK] the princess of the city passing by . ahmad falls in love with [MASK] and visits her in her garden . he tells leon he has come to her from beyond [MASK] [MASK] wins a [MASK] [MASK] then [MASK] is captured . when ja ##ffa ##r comes to win the princess of bas ##ra for himself , ahmad attacks the evil viz ##ier who blinds him and turns [SEP] ja ##ffa ##r [MASK] asks for the ##ap [MASK] s hand , and he gives the gift of a mechanical [MASK] horse [MASK] the sultan of bas ##ra . ##film blind [MASK] then tells his canonical in the [MASK] , accompanied by abu as his dog . [SEP]\n",
      "segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "is_random_next: False\n",
      "masked_lm_positions: 10 24 34 42 43 46 47 49 54 83 87 88 100 102 104 109 111 115 118\n",
      "masked_lm_labels: see her her time and kiss . he ja then princess ' flying to sultan the ahmad tale marketplace\n",
      "\n",
      "\n",
      "[INFO] number of train data: 1059\n",
      "[INFO] is_random_next ratio: 0.5684608120868744\n"
     ]
    }
   ],
   "source": [
    "# use spacy for sentence split\n",
    "nlp = spacy.load('en')\n",
    "\n",
    "# use IMDB movie review as pretraining data\n",
    "texts = train['sentence'].tolist() + test['sentence'].tolist()\n",
    "\n",
    "# To keep training fast on my macbook, I take a sample of 50 movie reviews. \n",
    "# Due to this operation, pretraining model may be very bad. \n",
    "# So you can try with all movie reviews to get better pre-training model.\n",
    "random.shuffle(texts)\n",
    "texts = texts[0:50]\n",
    "\n",
    "# sentence split\n",
    "sentences_texts=[]\n",
    "for text in texts:\n",
    "    doc = nlp(text)\n",
    "    sentences_texts.append([s.text for s in doc.sents])\n",
    "\n",
    "vocab_path = os.path.join(bert_data_path, 'vocab.txt')\n",
    "\n",
    "# set dupe_factor=5 to reduce the samples of pre-training data. defaut:10\n",
    "create_pretraining_data_from_docs(sentences_texts,\n",
    "                                  vocab_path=vocab_path,\n",
    "                                  save_path=os.path.join(bert_data_path,'pretraining_data.npz'),\n",
    "                                  token_method='wordpiece',\n",
    "                                  language='en',\n",
    "                                  dupe_factor=5\n",
    "                                 )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that we've prepared the pre-training data, let's focus on training a pre-training model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/zhongan/Documents/competition/BERT_with_keras/pretraining.py:107: UserWarning: model performance may be suitable when warmup steps is 0.01~0.02 of train steps.\n",
      "  warnings.warn(\"model performance may be suitable when warmup steps is 0.01~0.02 of train steps.\", UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "119/120 [============================>.] - ETA: 14s - loss: 3.3843 - lm_model_loss: 6.0340 - next_sentence_model_loss: 0.7346 - lm_model_acc: 0.2309 - next_sentence_model_acc: 0.5158\n",
      "step 0000120: cur_lm_acc is 0.25802, cur_is_random_nex_acc is 0.56604\n",
      "\n",
      "Step 0000120: val_acc improved from -inf to 0.41203, saving model to /Users/zhongan/Documents/competition/BERT_with_keras/models/bert_pretraining_movie_reviews.h5\n",
      "120/120 [==============================] - 1927s 16s/step - loss: 3.3834 - lm_model_loss: 6.0344 - next_sentence_model_loss: 0.7324 - lm_model_acc: 0.2299 - next_sentence_model_acc: 0.5198 - val_loss: 2.9744 - val_lm_model_loss: 5.2540 - val_next_sentence_model_loss: 0.6949 - val_lm_model_acc: 0.2580 - val_next_sentence_model_acc: 0.5660\n",
      "Epoch 2/2\n",
      "110/120 [==========================>...] - ETA: 2:19 - loss: 2.9011 - lm_model_loss: 5.1127 - next_sentence_model_loss: 0.6895 - lm_model_acc: 0.2390 - next_sentence_model_acc: 0.5659\n",
      "step 0000230: cur_lm_acc is 0.25142, cur_is_random_nex_acc is 0.56604\n",
      "\n",
      "Step 0000230: val_acc did not improve from 0.41203, current is 0.40873\n",
      "120/120 [==============================] - 1779s 15s/step - loss: 2.8996 - lm_model_loss: 5.1107 - next_sentence_model_loss: 0.6886 - lm_model_acc: 0.2391 - next_sentence_model_acc: 0.5698 - val_loss: 2.9544 - val_lm_model_loss: 5.2242 - val_next_sentence_model_loss: 0.6846 - val_lm_model_acc: 0.2514 - val_next_sentence_model_acc: 0.5660\n"
     ]
    }
   ],
   "source": [
    "# pretraining a bert encoder model\n",
    "bert_pretraining(train_data_path=os.path.join(bert_data_path,'pretraining_data.npz'),\n",
    "                 bert_config_file=os.path.join(bert_data_path, 'bert_config.json'),\n",
    "                 save_path=bert_model_path,\n",
    "                 batch_size=8,\n",
    "                 seq_length=128,\n",
    "                 max_predictions_per_seq=20,\n",
    "                 val_batch_size=128,\n",
    "                 multi_gpu=0,\n",
    "                 num_warmup_steps=10,\n",
    "                 checkpoints_interval_steps=110,\n",
    "                 max_num_val=1000,\n",
    "                 pretraining_model_name='bert_pretraining_movie_reviews.h5',\n",
    "                 encoder_model_name='bert_movie_reviews_encoder.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fune-tuning\n",
    "Use the above pre-training model as the initial point for your NLP model.Here the pre-training model is used to classify the movie reviews(i.e. classifying whether a movie review is positive or negtive)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import keras\n",
    "import numpy as np\n",
    "from const import bert_data_path, bert_model_path\n",
    "from modeling import BertConfig\n",
    "from classifier import SingleSeqDataProcessor, convert_examples_to_features, Text_Classifier, save_features, TextSequence\n",
    "from tokenization import FullTokenizer\n",
    "from optimization import AdamWeightDecayOpt\n",
    "from checkpoint import StepModelCheckpoint"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, we need to convert movie reviews data to what we need in bert."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** Example ***\n",
      "guid: train-0\n",
      "tokens: [CLS] it was a good story , but not very well told . i liked the themes and the main story line , which wasn ' t as clear as it could have been . maybe there was too much going on and a lack of ability to reign it all in . the acting was okay to che ##es ##y , some were stronger than others and even the stronger actors had their moments of lesser quality acting . it took me a couple of months to get through the entire movie because it didn ' t keep my attention and the flow was just bad . i only just finished watching it and i ' m glad i did as the movie finally gets moving [SEP]\n",
      "input_ids: 101 2009 2001 1037 2204 2466 1010 2021 2025 2200 2092 2409 1012 1045 4669 1996 6991 1998 1996 2364 2466 2240 1010 2029 2347 1005 1056 2004 3154 2004 2009 2071 2031 2042 1012 2672 2045 2001 2205 2172 2183 2006 1998 1037 3768 1997 3754 2000 5853 2009 2035 1999 1012 1996 3772 2001 3100 2000 18178 2229 2100 1010 2070 2020 6428 2084 2500 1998 2130 1996 6428 5889 2018 2037 5312 1997 8276 3737 3772 1012 2009 2165 2033 1037 3232 1997 2706 2000 2131 2083 1996 2972 3185 2138 2009 2134 1005 1056 2562 2026 3086 1998 1996 4834 2001 2074 2919 1012 1045 2069 2074 2736 3666 2009 1998 1045 1005 1049 5580 1045 2106 2004 1996 3185 2633 4152 3048 102\n",
      "input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "label: 0 (id = 0)\n",
      "*** Example ***\n",
      "guid: train-1\n",
      "tokens: [CLS] after eras ##ing my thoughts nearly twenty - seven times , there is a feeling that i can now conquer this review for the complex french drama , \" read my lips \" . having written over five hundred reviews , i have never found myself at such a loss of words as i did with director jacques audi ##ard ' s subtle , yet inspirational love story . thought was poured over what was loved and hated about this film , and while the \" loves \" over ##powered , it was the elements that were hated that sparked further debate within my mind . \" read my lips \" is a drama . to be more precise , is a character driven drama which [SEP]\n",
      "input_ids: 101 2044 28500 2075 2026 4301 3053 3174 1011 2698 2335 1010 2045 2003 1037 3110 2008 1045 2064 2085 16152 2023 3319 2005 1996 3375 2413 3689 1010 1000 3191 2026 2970 1000 1012 2383 2517 2058 2274 3634 4391 1010 1045 2031 2196 2179 2870 2012 2107 1037 3279 1997 2616 2004 1045 2106 2007 2472 7445 20075 4232 1005 1055 11259 1010 2664 28676 2293 2466 1012 2245 2001 8542 2058 2054 2001 3866 1998 6283 2055 2023 2143 1010 1998 2096 1996 1000 7459 1000 2058 27267 1010 2009 2001 1996 3787 2008 2020 6283 2008 13977 2582 5981 2306 2026 2568 1012 1000 3191 2026 2970 1000 2003 1037 3689 1012 2000 2022 2062 10480 1010 2003 1037 2839 5533 3689 2029 102\n",
      "input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "label: 1 (id = 1)\n",
      "*** Example ***\n",
      "guid: train-2\n",
      "tokens: [CLS] come on tina fey you can do better then this . as soon as the movie started i knew how it would end . sure it was funny at times . even laugh out loud funny . but there isn ' t enough laughs to save this movie . i don ' t recommend buying this . at the most i recommend rent ##ing it but that ##s all . baby mama has some funny scenes but is predictable and fails to have the heart ##war ##ming ending it strive ##s for . < br / > < br / > tina fey and amy po ##al ##her made a good team . mean girls is one of my favorite movies . tina fey and amy [SEP]\n",
      "input_ids: 101 2272 2006 11958 23864 2017 2064 2079 2488 2059 2023 1012 2004 2574 2004 1996 3185 2318 1045 2354 2129 2009 2052 2203 1012 2469 2009 2001 6057 2012 2335 1012 2130 4756 2041 5189 6057 1012 2021 2045 3475 1005 1056 2438 11680 2000 3828 2023 3185 1012 1045 2123 1005 1056 16755 9343 2023 1012 2012 1996 2087 1045 16755 9278 2075 2009 2021 2008 2015 2035 1012 3336 9588 2038 2070 6057 5019 2021 2003 21425 1998 11896 2000 2031 1996 2540 9028 6562 4566 2009 29453 2015 2005 1012 1026 7987 1013 1028 1026 7987 1013 1028 11958 23864 1998 6864 13433 2389 5886 2081 1037 2204 2136 1012 2812 3057 2003 2028 1997 2026 5440 5691 1012 11958 23864 1998 6864 102\n",
      "input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "label: 0 (id = 0)\n",
      "*** Example ***\n",
      "guid: train-3\n",
      "tokens: [CLS] i read somewhere where this film was supposed to be a remake of the 1949 film noir , \" cr ##iss cross . \" i found the latter to be disappointing but it was still better than this film . < br / > < br / > this movie is a \" neo - noir \" since it ' s modern - day and it ' s in color , two things that pu ##rist ##s would make it be disqualified for film noir status . < br / > < br / > the biggest negative to it , however , wasn ' t the cinematography ( that was fine ) but the mud ##dled storyline . hey , some of ' 40 ##s [SEP]\n",
      "input_ids: 101 1045 3191 4873 2073 2023 2143 2001 4011 2000 2022 1037 12661 1997 1996 4085 2143 15587 1010 1000 13675 14643 2892 1012 1000 1045 2179 1996 3732 2000 2022 15640 2021 2009 2001 2145 2488 2084 2023 2143 1012 1026 7987 1013 1028 1026 7987 1013 1028 2023 3185 2003 1037 1000 9253 1011 15587 1000 2144 2009 1005 1055 2715 1011 2154 1998 2009 1005 1055 1999 3609 1010 2048 2477 2008 16405 15061 2015 2052 2191 2009 2022 14209 2005 2143 15587 3570 1012 1026 7987 1013 1028 1026 7987 1013 1028 1996 5221 4997 2000 2009 1010 2174 1010 2347 1005 1056 1996 16434 1006 2008 2001 2986 1007 2021 1996 8494 20043 9994 1012 4931 1010 2070 1997 1005 2871 2015 102\n",
      "input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "label: 0 (id = 0)\n",
      "*** Example ***\n",
      "guid: train-4\n",
      "tokens: [CLS] this movie will kick your ass ! powerful acting in a story that pushes all of us to live out our dreams . jake g ##yl ##len ##ha ##al will go places from here , and the supporting cast was superb . why would would anyone want to stay in coal ##ville and develop black lung anyway ? [SEP]\n",
      "input_ids: 101 2023 3185 2097 5926 2115 4632 999 3928 3772 1999 1037 2466 2008 13956 2035 1997 2149 2000 2444 2041 2256 5544 1012 5180 1043 8516 7770 3270 2389 2097 2175 3182 2013 2182 1010 1998 1996 4637 3459 2001 21688 1012 2339 2052 2052 3087 2215 2000 2994 1999 5317 3077 1998 4503 2304 11192 4312 1029 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "label: 1 (id = 1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** Example ***\n",
      "guid: dev-0\n",
      "tokens: [CLS] is it just me or are most of the actors in this adaptation mis ##cast age - wise ? < br / > < br / > jem ##ma red ##grave , although a superior actress , seems as if she should be joining her children instead of nodding off on the couch . ( i would have loved to see brenda b ##let ##hly ##n in the part , her connection to the latest pride and prejudice notwithstanding . ) < br / > < br / > blake ri ##tson and rory kin ##nea ##r were the only two whom i felt lived up to their characters in spirit and performance . every else looked confused or out of place . < br / [SEP]\n",
      "input_ids: 101 2003 2009 2074 2033 2030 2024 2087 1997 1996 5889 1999 2023 6789 28616 10526 2287 1011 7968 1029 1026 7987 1013 1028 1026 7987 1013 1028 24193 2863 2417 12830 1010 2348 1037 6020 3883 1010 3849 2004 2065 2016 2323 2022 5241 2014 2336 2612 1997 11863 2125 2006 1996 6411 1012 1006 1045 2052 2031 3866 2000 2156 15507 1038 7485 27732 2078 1999 1996 2112 1010 2014 4434 2000 1996 6745 6620 1998 18024 26206 1012 1007 1026 7987 1013 1028 1026 7987 1013 1028 6511 15544 25656 1998 14285 12631 22084 2099 2020 1996 2069 2048 3183 1045 2371 2973 2039 2000 2037 3494 1999 4382 1998 2836 1012 2296 2842 2246 5457 2030 2041 1997 2173 1012 1026 7987 1013 102\n",
      "input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "label: 0 (id = 0)\n",
      "*** Example ***\n",
      "guid: dev-1\n",
      "tokens: [CLS] i never saw doctor who before ( at least not in any focused way ) , so i was new to the concept . i have to say that the new show works very well . it ' s funny ( it really also ought to say \" comedy \" in the genre description ; many plot turns are only acceptable because of their comedic value ) , it ' s well - written and it ' s making a me ##ager budget go a long way . the human dimension is very strong and engaging , which is very rare in current tv shows . < br / > < br / > i ' ve seen the first eight episodes , and # 6 [SEP]\n",
      "input_ids: 101 1045 2196 2387 3460 2040 2077 1006 2012 2560 2025 1999 2151 4208 2126 1007 1010 2061 1045 2001 2047 2000 1996 4145 1012 1045 2031 2000 2360 2008 1996 2047 2265 2573 2200 2092 1012 2009 1005 1055 6057 1006 2009 2428 2036 11276 2000 2360 1000 4038 1000 1999 1996 6907 6412 1025 2116 5436 4332 2024 2069 11701 2138 1997 2037 21699 3643 1007 1010 2009 1005 1055 2092 1011 2517 1998 2009 1005 1055 2437 1037 2033 17325 5166 2175 1037 2146 2126 1012 1996 2529 9812 2003 2200 2844 1998 11973 1010 2029 2003 2200 4678 1999 2783 2694 3065 1012 1026 7987 1013 1028 1026 7987 1013 1028 1045 1005 2310 2464 1996 2034 2809 4178 1010 1998 1001 1020 102\n",
      "input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "label: 1 (id = 1)\n",
      "*** Example ***\n",
      "guid: dev-2\n",
      "tokens: [CLS] this is another notorious mexican horror film : however , while the original spanish - language version is quite tame , all sorts of gore and nu ##dity were inserted for the english - dubbed variant ( prepared by je ##ral ##d intra ##tor - who did similar duties on the curious dr . hum ##pp [ 1967 / 71 ] - and , like the u . s . version of that film , had previously been available on dvd through something weird / image ) ! < br / > < br / > i watched the original first and , while no great shakes , it was fairly engaging - especially with a plot as familiar as this one was : a doctor [SEP]\n",
      "input_ids: 101 2023 2003 2178 12536 4916 5469 2143 1024 2174 1010 2096 1996 2434 3009 1011 2653 2544 2003 3243 24763 1010 2035 11901 1997 13638 1998 16371 25469 2020 12889 2005 1996 2394 1011 9188 8349 1006 4810 2011 15333 7941 2094 26721 4263 1011 2040 2106 2714 5704 2006 1996 8025 2852 1012 14910 9397 1031 3476 1013 6390 1033 1011 1998 1010 2066 1996 1057 1012 1055 1012 2544 1997 2008 2143 1010 2018 3130 2042 2800 2006 4966 2083 2242 6881 1013 3746 1007 999 1026 7987 1013 1028 1026 7987 1013 1028 1045 3427 1996 2434 2034 1998 1010 2096 2053 2307 10854 1010 2009 2001 7199 11973 1011 2926 2007 1037 5436 2004 5220 2004 2023 2028 2001 1024 1037 3460 102\n",
      "input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "label: 0 (id = 0)\n",
      "*** Example ***\n",
      "guid: dev-3\n",
      "tokens: [CLS] house calls was an amusing 1978 comedy about a widowed doctor ( walter matt ##ha ##u ) who now wants to play the field but can ' t help but be drawn to a patient of his ( glen ##da jackson ) who refuses to be just another notch on his bed ##post . matt ##ha ##u likes the woman but does not really want to make the commitment that she insists upon so he agrees to date her exclusively for two weeks and then make a decision as to whether or not he wants to commit ; however , other complications make it difficult for matt ##ha ##u to make a decision when the two weeks are up , even though he is clearly in [SEP]\n",
      "input_ids: 101 2160 4455 2001 2019 19142 3301 4038 2055 1037 22874 3460 1006 4787 4717 3270 2226 1007 2040 2085 4122 2000 2377 1996 2492 2021 2064 1005 1056 2393 2021 2022 4567 2000 1037 5776 1997 2010 1006 8904 2850 4027 1007 2040 10220 2000 2022 2074 2178 18624 2006 2010 2793 19894 1012 4717 3270 2226 7777 1996 2450 2021 2515 2025 2428 2215 2000 2191 1996 8426 2008 2016 16818 2588 2061 2002 10217 2000 3058 2014 7580 2005 2048 3134 1998 2059 2191 1037 3247 2004 2000 3251 2030 2025 2002 4122 2000 10797 1025 2174 1010 2060 12763 2191 2009 3697 2005 4717 3270 2226 2000 2191 1037 3247 2043 1996 2048 3134 2024 2039 1010 2130 2295 2002 2003 4415 1999 102\n",
      "input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "label: 1 (id = 1)\n",
      "*** Example ***\n",
      "guid: dev-4\n",
      "tokens: [CLS] i stumbled on to this site while looking for a video or dvd of the 1959 version por ##gy and be ##ss with sammy davis as sport ##in ' life . if anyone finds this on a home movie format please let me know . i talk to my daughters all the time about things that they think are new which , actually have already been done . we went to see a live theater of version a couple of years ago and all i could talk about was this film . sadly my daughters cannot remember seeing sammy davis jr . in any production , although they have heard of him . needles ##s to say , they ' re not familiar with the other [SEP]\n",
      "input_ids: 101 1045 9845 2006 2000 2023 2609 2096 2559 2005 1037 2678 2030 4966 1997 1996 3851 2544 18499 6292 1998 2022 4757 2007 14450 4482 2004 4368 2378 1005 2166 1012 2065 3087 4858 2023 2006 1037 2188 3185 4289 3531 2292 2033 2113 1012 1045 2831 2000 2026 5727 2035 1996 2051 2055 2477 2008 2027 2228 2024 2047 2029 1010 2941 2031 2525 2042 2589 1012 2057 2253 2000 2156 1037 2444 4258 1997 2544 1037 3232 1997 2086 3283 1998 2035 1045 2071 2831 2055 2001 2023 2143 1012 13718 2026 5727 3685 3342 3773 14450 4482 3781 1012 1999 2151 2537 1010 2348 2027 2031 2657 1997 2032 1012 17044 2015 2000 2360 1010 2027 1005 2128 2025 5220 2007 1996 2060 102\n",
      "input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "label: 1 (id = 1)\n"
     ]
    }
   ],
   "source": [
    "train_examples = SingleSeqDataProcessor.get_train_examples(train_data=train['sentence'].tolist(),labels=train['polarity'].tolist())\n",
    "dev_exmaples = SingleSeqDataProcessor.get_dev_examples(dev_data=test['sentence'].tolist(), labels=test['polarity'].tolist())\n",
    "\n",
    "# `word piece tokenizer` need to a prepared vocabulary.\n",
    "vocab_path = os.path.join(bert_data_path, 'vocab.txt')\n",
    "\n",
    "# load vocab to tokenizer\n",
    "tokenizer = FullTokenizer(vocab_path, do_lower_case=True)\n",
    "\n",
    "# convert the train and dev examples to features\n",
    "train_features = convert_examples_to_features(train_examples, \n",
    "                                              label_list=[0,1], \n",
    "                                              max_seq_length=128, \n",
    "                                              tokenizer= tokenizer)\n",
    "dev_features = convert_examples_to_features(dev_exmaples, label_list=[0,1], max_seq_length=128, tokenizer=tokenizer)\n",
    "\n",
    "# convert features to a dictionary of numpy arrays.\n",
    "train_features_array_dict = save_features(features=train_features)\n",
    "dev_features_array_dict = save_features(features=dev_features)\n",
    "\n",
    "# get train and validation data\n",
    "train_x = [train_features_array_dict['input_ids'], train_features_array_dict['input_mask'], train_features_array_dict['segment_ids']]\n",
    "train_y = keras.utils.to_categorical(train_features_array_dict['label_ids'], 2)\n",
    "val_x = [dev_features_array_dict['input_ids'], dev_features_array_dict['input_mask'], dev_features_array_dict['segment_ids']]\n",
    "val_y = keras.utils.to_categorical(dev_features_array_dict['label_ids'],2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Secend, we need to set the args of bert model. we also need to define a opimizer and a checkponter."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of train steps: 4689\n"
     ]
    }
   ],
   "source": [
    "# load bert configuration file\n",
    "config = BertConfig.from_json_file(os.path.join(bert_data_path, 'bert_config.json'))\n",
    "epochs = 3\n",
    "num_gpus = None\n",
    "# if you come across a OOM problem, reduce the batch size.\n",
    "batch_size = 16\n",
    "\n",
    "# calculation the number of training steps by epoch size.\n",
    "num_train_samples = len(train_features_array_dict['input_ids'])\n",
    "num_train_steps = int(np.ceil(num_train_samples / batch_size)) * epochs\n",
    "print(\"number of train steps: {}\".format(num_train_steps))\n",
    "\n",
    "# Use weight decay adam optimizer. this optimizer is sightly different with Keras's Standard Adam optimizer. \n",
    "# For more details, view source code of AdamWeightDecayOpt.\n",
    "adam = AdamWeightDecayOpt(\n",
    "        lr=5e-5,\n",
    "        num_train_steps=num_train_steps,\n",
    "        num_warmup_steps=100,\n",
    "        beta_1=0.9,\n",
    "        beta_2=0.999,\n",
    "        epsilon=1e-6,\n",
    "        weight_decay_rate=0.01,\n",
    "        exclude_from_weight_decay=[\"LayerNorm\", \"layer_norm\", \"bias\"]\n",
    "    )\n",
    "\n",
    "# This checkpoint evaluate the bert model performance on batch end.\n",
    "checkpoint = StepModelCheckpoint(filepath=\"%s/%s\" % (bert_model_path, 'imdb_classifer_model.h5'),\n",
    "                                 verbose=1, monitor='val_acc',\n",
    "                                 save_best_only=True,\n",
    "                                 xlen=3,\n",
    "                                 period=100,\n",
    "                                 start_step=100,\n",
    "                                 val_batch_size=128)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Last, create a bert classification model and train it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n",
      "  16/1562 [..............................] - ETA: 6:19:34 - loss: 0.7176 - acc: 0.5430"
     ]
    }
   ],
   "source": [
    "# create a model\n",
    "classifier = Text_Classifier(bert_config=config,\n",
    "                             pretrain_model_path=os.path.join(bert_model_path, 'bert_movie_reviews_encoder.h5'),\n",
    "                             batch_size=batch_size,\n",
    "                             seq_length=128,\n",
    "                             optimizer=adam,\n",
    "                             num_classes=2,\n",
    "                             metrics=['acc'],\n",
    "                             multi_gpu= num_gpus\n",
    "                             )\n",
    "\n",
    "# when using multi-gpus, the parallel model of bert cann't be used to evaluate/predict.\n",
    "# You can only use the cpu_build model to evalate and predict.\n",
    "if num_gpus is not None:\n",
    "    checkpoint.single_gpu_model = classifier.model\n",
    "\n",
    "# train model\n",
    "generator= TextSequence(x=train_x,y=train_y,batch_size=batch_size)\n",
    "history = classifier.fit_generator(generator=generator,\n",
    "                                   epochs=epochs,\n",
    "                                   shuffle=True,\n",
    "                                   callbacks=[checkpoint],\n",
    "                                   validation_data=(val_x,val_y)\n",
    "                                   )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
